import json
import numpy as np
from gensim.models import Word2Vec
from sklearn.preprocessing import MultiLabelBinarizer
from sklearn.model_selection import train_test_split
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv1D, GlobalMaxPooling1D, Dense, Embedding, Dropout
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.preprocessing.text import Tokenizer

# 加载数据
file_path = r"D:\研一\课程\自然语言处理\data\data\train.json"
texts = []
labels = []

with open(file_path, 'r', encoding='utf-8') as f:
    for line in f:
        data = json.loads(line.strip())
        texts.append(data['text'])
        labels.append(data['label'])

# 分词
texts_tokenized = [text.split() for text in texts]

# 生成Word2Vec词向量
w2v_model = Word2Vec(sentences=texts_tokenized, vector_size=100, window=5, min_count=2, workers=4)
embedding_matrix = np.zeros((len(w2v_model.wv), 100))
for i in range(len(w2v_model.wv)):
    embedding_matrix[i] = w2v_model.wv[w2v_model.wv.index_to_key[i]]

# 将文本转换为序列
tokenizer = Tokenizer()
tokenizer.fit_on_texts(texts)
X = tokenizer.texts_to_sequences(texts)

# 填充序列，使其长度一致
max_length = 100  # 可根据需求调整
X = pad_sequences(X, maxlen=max_length, padding='post')

# 标签二进制化
mlb = MultiLabelBinarizer()
Y = mlb.fit_transform(labels)

# 划分训练集和测试集
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=42)

# 构建TextCNN模型
model = Sequential()
model.add(Embedding(input_dim=len(w2v_model.wv), output_dim=100, weights=[embedding_matrix], input_length=max_length, trainable=False))
model.add(Conv1D(128, kernel_size=3, activation='relu'))
model.add(GlobalMaxPooling1D())
model.add(Dropout(0.5))
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(len(mlb.classes_), activation='sigmoid'))  # 使用sigmoid以适应多标签分类

# 编译模型
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])

# 训练模型
model.fit(X_train, Y_train, epochs=10, batch_size=32, validation_data=(X_test, Y_test))

# 评估模型
loss, accuracy = model.evaluate(X_test, Y_test)
print("Test Accuracy:", accuracy)
